# -*- coding: utf-8 -*-

BOT_NAME = 'dp_crawler'

SPIDER_MODULES = ['dp_crawler.spiders']
NEWSPIDER_MODULE = 'dp_crawler.spiders'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

LOG_LEVEL = "INFO"

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
    "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None,
    "scrapy.downloadermiddlewares.retry.RetryMiddleware": None,
    "dp_crawler.middleware.downloader_middleware.rotate_agent.RotateUserAgentMiddleware": 400,
    "dp_crawler.middleware.downloader_middleware.proxy.ProxyMiddleware": 600
}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'dp_crawler.pipelines.DpCrawlerPipeline': 300,
}

DOWNLOAD_TIMEOUT = 10
